{
 "nbformat": 4,
 "nbformat_minor": 2,
 "metadata": {
  "language_info": {
   "name": "python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "version": "3.8.1-final"
  },
  "orig_nbformat": 2,
  "file_extension": ".py",
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3,
  "kernelspec": {
   "name": "python38164bitec4538a0ed7a4029b9bd19594323cc7e",
   "display_name": "Python 3.8.1 64-bit"
  }
 },
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>movie_id</th>\n      <th>genres</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>4</td>\n      <td>Comedy|Drama|Romance</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>8</td>\n      <td>Adventure|Children</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>9</td>\n      <td>Action</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>12</td>\n      <td>Comedy|Horror</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>13</td>\n      <td>Adventure|Animation|Children</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2605</th>\n      <td>112183</td>\n      <td>Comedy|Drama</td>\n    </tr>\n    <tr>\n      <th>2606</th>\n      <td>112290</td>\n      <td>Drama</td>\n    </tr>\n    <tr>\n      <th>2607</th>\n      <td>112556</td>\n      <td>Drama|Thriller</td>\n    </tr>\n    <tr>\n      <th>2608</th>\n      <td>112852</td>\n      <td>Action|Adventure|Sci-Fi</td>\n    </tr>\n    <tr>\n      <th>2609</th>\n      <td>116797</td>\n      <td>Drama|Thriller</td>\n    </tr>\n  </tbody>\n</table>\n<p>2610 rows × 2 columns</p>\n</div>",
      "text/plain": "      movie_id                        genres\n0            4          Comedy|Drama|Romance\n1            8            Adventure|Children\n2            9                        Action\n3           12                 Comedy|Horror\n4           13  Adventure|Animation|Children\n...        ...                           ...\n2605    112183                  Comedy|Drama\n2606    112290                         Drama\n2607    112556                Drama|Thriller\n2608    112852       Action|Adventure|Sci-Fi\n2609    116797                Drama|Thriller\n\n[2610 rows x 2 columns]"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_old =pd.read_csv('./data/movie_old.csv', usecols=[0,2])\n",
    "df_old"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "{'Action',\n 'Adventure',\n 'Animation',\n 'Children',\n 'Comedy',\n 'Crime',\n 'Documentary',\n 'Drama',\n 'Fantasy',\n 'Film-Noir',\n 'Horror',\n 'IMAX',\n 'Musical',\n 'Mystery',\n 'Romance',\n 'Sci-Fi',\n 'Thriller',\n 'War',\n 'Western'}"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_genres = set((el for g in df_old.genres for el in g.split('|')))\n",
    "all_genres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "{'War': 0,\n 'Mystery': 1,\n 'Documentary': 2,\n 'Action': 3,\n 'Comedy': 4,\n 'Drama': 5,\n 'IMAX': 6,\n 'Western': 7,\n 'Musical': 8,\n 'Animation': 9,\n 'Fantasy': 10,\n 'Thriller': 11,\n 'Horror': 12,\n 'Sci-Fi': 13,\n 'Romance': 14,\n 'Film-Noir': 15,\n 'Adventure': 16,\n 'Children': 17,\n 'Crime': 18}"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_genres = list(all_genres)\n",
    "all_genres_map = dict(\n",
    "    ((all_genres[i],i) for i in range(len(all_genres)))\n",
    ")\n",
    "all_genres_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>movie_id</th>\n      <th>genres</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>14</td>\n      <td>Drama</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>18</td>\n      <td>Comedy</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>30</td>\n      <td>Crime|Drama</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>35</td>\n      <td>Drama|Romance</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>41</td>\n      <td>Drama|War</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>851</th>\n      <td>103341</td>\n      <td>Action|Comedy|Sci-Fi</td>\n    </tr>\n    <tr>\n      <th>852</th>\n      <td>104879</td>\n      <td>Drama|Mystery|Thriller</td>\n    </tr>\n    <tr>\n      <th>853</th>\n      <td>104913</td>\n      <td>Action|Drama</td>\n    </tr>\n    <tr>\n      <th>854</th>\n      <td>105844</td>\n      <td>Drama</td>\n    </tr>\n    <tr>\n      <th>855</th>\n      <td>112623</td>\n      <td>Sci-Fi</td>\n    </tr>\n  </tbody>\n</table>\n<p>856 rows × 2 columns</p>\n</div>",
      "text/plain": "     movie_id                  genres\n0          14                   Drama\n1          18                  Comedy\n2          30             Crime|Drama\n3          35           Drama|Romance\n4          41               Drama|War\n..        ...                     ...\n851    103341    Action|Comedy|Sci-Fi\n852    104879  Drama|Mystery|Thriller\n853    104913            Action|Drama\n854    105844                   Drama\n855    112623                  Sci-Fi\n\n[856 rows x 2 columns]"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_new =pd.read_csv('./data/movie_new.csv', usecols=[0,2])\n",
    "df_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "(0, 14)\n"
    }
   ],
   "source": [
    "for i, (mid, _) in df_new.iterrows(): \n",
    "    print ((i, mid))\n",
    "    break \n",
    "# def getId2Index(df:pd.DataFrame):\n",
    "#     return dict(*(mid, index) for i in df. )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getId2Index(df:pd.DataFrame):\n",
    "    return dict(((mid, i) for i, (mid,_) in df.iterrows()))\n",
    "df_old_id_map = getId2Index(df_old)\n",
    "df_new_id_map = getId2Index(df_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "{14: 0,\n 18: 1,\n 30: 2,\n 35: 3,\n 41: 4,\n 46: 5,\n 64: 6,\n 65: 7,\n 69: 8,\n 72: 9,\n 73: 10,\n 74: 11,\n 76: 12,\n 79: 13,\n 82: 14,\n 86: 15,\n 100: 16,\n 107: 17,\n 113: 18,\n 121: 19,\n 122: 20,\n 132: 21,\n 135: 22,\n 140: 23,\n 144: 24,\n 157: 25,\n 162: 26,\n 174: 27,\n 175: 28,\n 176: 29,\n 187: 30,\n 194: 31,\n 195: 32,\n 204: 33,\n 207: 34,\n 214: 35,\n 222: 36,\n 237: 37,\n 242: 38,\n 255: 39,\n 257: 40,\n 258: 41,\n 267: 42,\n 272: 43,\n 273: 44,\n 276: 45,\n 290: 46,\n 307: 47,\n 325: 48,\n 338: 49,\n 362: 50,\n 365: 51,\n 369: 52,\n 371: 53,\n 373: 54,\n 379: 55,\n 383: 56,\n 405: 57,\n 412: 58,\n 413: 59,\n 415: 60,\n 423: 61,\n 427: 62,\n 428: 63,\n 433: 64,\n 450: 65,\n 452: 66,\n 459: 67,\n 464: 68,\n 468: 69,\n 481: 70,\n 513: 71,\n 522: 72,\n 531: 73,\n 532: 74,\n 533: 75,\n 534: 76,\n 537: 77,\n 542: 78,\n 546: 79,\n 548: 80,\n 558: 81,\n 611: 82,\n 613: 83,\n 665: 84,\n 671: 85,\n 678: 86,\n 688: 87,\n 695: 88,\n 709: 89,\n 728: 90,\n 737: 91,\n 765: 92,\n 801: 93,\n 809: 94,\n 830: 95,\n 836: 96,\n 839: 97,\n 842: 98,\n 880: 99,\n 886: 100,\n 888: 101,\n 909: 102,\n 911: 103,\n 916: 104,\n 918: 105,\n 921: 106,\n 928: 107,\n 930: 108,\n 933: 109,\n 940: 110,\n 945: 111,\n 949: 112,\n 950: 113,\n 965: 114,\n 971: 115,\n 991: 116,\n 998: 117,\n 1004: 118,\n 1011: 119,\n 1016: 120,\n 1019: 121,\n 1023: 122,\n 1025: 123,\n 1030: 124,\n 1033: 125,\n 1041: 126,\n 1043: 127,\n 1057: 128,\n 1066: 129,\n 1078: 130,\n 1081: 131,\n 1086: 132,\n 1091: 133,\n 1096: 134,\n 1100: 135,\n 1103: 136,\n 1104: 137,\n 1124: 138,\n 1125: 139,\n 1131: 140,\n 1147: 141,\n 1171: 142,\n 1177: 143,\n 1178: 144,\n 1185: 145,\n 1189: 146,\n 1194: 147,\n 1217: 148,\n 1218: 149,\n 1227: 150,\n 1237: 151,\n 1238: 152,\n 1241: 153,\n 1248: 154,\n 1251: 155,\n 1256: 156,\n 1257: 157,\n 1260: 158,\n 1264: 159,\n 1277: 160,\n 1280: 161,\n 1283: 162,\n 1289: 163,\n 1290: 164,\n 1297: 165,\n 1301: 166,\n 1303: 167,\n 1348: 168,\n 1350: 169,\n 1381: 170,\n 1390: 171,\n 1392: 172,\n 1398: 173,\n 1399: 174,\n 1417: 175,\n 1419: 176,\n 1425: 177,\n 1449: 178,\n 1464: 179,\n 1465: 180,\n 1476: 181,\n 1480: 182,\n 1483: 183,\n 1499: 184,\n 1518: 185,\n 1541: 186,\n 1554: 187,\n 1556: 188,\n 1588: 189,\n 1598: 190,\n 1600: 191,\n 1603: 192,\n 1604: 193,\n 1606: 194,\n 1611: 195,\n 1616: 196,\n 1619: 197,\n 1620: 198,\n 1633: 199,\n 1635: 200,\n 1665: 201,\n 1672: 202,\n 1678: 203,\n 1687: 204,\n 1701: 205,\n 1713: 206,\n 1719: 207,\n 1726: 208,\n 1727: 209,\n 1734: 210,\n 1769: 211,\n 1772: 212,\n 1792: 213,\n 1810: 214,\n 1834: 215,\n 1840: 216,\n 1841: 217,\n 1845: 218,\n 1863: 219,\n 1885: 220,\n 1894: 221,\n 1897: 222,\n 1911: 223,\n 1914: 224,\n 1945: 225,\n 1949: 226,\n 1955: 227,\n 1956: 228,\n 1958: 229,\n 1964: 230,\n 1965: 231,\n 1971: 232,\n 1974: 233,\n 1986: 234,\n 1998: 235,\n 2010: 236,\n 2013: 237,\n 2015: 238,\n 2025: 239,\n 2034: 240,\n 2042: 241,\n 2048: 242,\n 2052: 243,\n 2053: 244,\n 2070: 245,\n 2073: 246,\n 2082: 247,\n 2088: 248,\n 2089: 249,\n 2090: 250,\n 2093: 251,\n 2102: 252,\n 2112: 253,\n 2116: 254,\n 2121: 255,\n 2122: 256,\n 2132: 257,\n 2137: 258,\n 2138: 259,\n 2141: 260,\n 2143: 261,\n 2146: 262,\n 2147: 263,\n 2153: 264,\n 2173: 265,\n 2195: 266,\n 2206: 267,\n 2240: 268,\n 2247: 269,\n 2259: 270,\n 2262: 271,\n 2263: 272,\n 2266: 273,\n 2290: 274,\n 2296: 275,\n 2297: 276,\n 2306: 277,\n 2312: 278,\n 2318: 279,\n 2331: 280,\n 2333: 281,\n 2340: 282,\n 2357: 283,\n 2363: 284,\n 2367: 285,\n 2375: 286,\n 2377: 287,\n 2380: 288,\n 2381: 289,\n 2383: 290,\n 2384: 291,\n 2385: 292,\n 2398: 293,\n 2401: 294,\n 2402: 295,\n 2403: 296,\n 2414: 297,\n 2428: 298,\n 2431: 299,\n 2433: 300,\n 2439: 301,\n 2442: 302,\n 2445: 303,\n 2447: 304,\n 2450: 305,\n 2457: 306,\n 2463: 307,\n 2467: 308,\n 2469: 309,\n 2473: 310,\n 2474: 311,\n 2476: 312,\n 2485: 313,\n 2511: 314,\n 2518: 315,\n 2520: 316,\n 2522: 317,\n 2524: 318,\n 2531: 319,\n 2533: 320,\n 2546: 321,\n 2550: 322,\n 2553: 323,\n 2558: 324,\n 2560: 325,\n 2567: 326,\n 2574: 327,\n 2575: 328,\n 2587: 329,\n 2594: 330,\n 2600: 331,\n 2613: 332,\n 2624: 333,\n 2648: 334,\n 2664: 335,\n 2677: 336,\n 2686: 337,\n 2687: 338,\n 2688: 339,\n 2707: 340,\n 2718: 341,\n 2725: 342,\n 2726: 343,\n 2728: 344,\n 2729: 345,\n 2732: 346,\n 2748: 347,\n 2749: 348,\n 2750: 349,\n 2759: 350,\n 2764: 351,\n 2792: 352,\n 2798: 353,\n 2806: 354,\n 2807: 355,\n 2819: 356,\n 2822: 357,\n 2827: 358,\n 2829: 359,\n 2840: 360,\n 2841: 361,\n 2846: 362,\n 2851: 363,\n 2861: 364,\n 2870: 365,\n 2875: 366,\n 2877: 367,\n 2883: 368,\n 2889: 369,\n 2905: 370,\n 2906: 371,\n 2907: 372,\n 2919: 373,\n 2920: 374,\n 2924: 375,\n 2929: 376,\n 2935: 377,\n 2940: 378,\n 2942: 379,\n 2946: 380,\n 2967: 381,\n 2969: 382,\n 2976: 383,\n 2989: 384,\n 2991: 385,\n 3005: 386,\n 3016: 387,\n 3019: 388,\n 3030: 389,\n 3040: 390,\n 3044: 391,\n 3060: 392,\n 3062: 393,\n 3071: 394,\n 3083: 395,\n 3095: 396,\n 3097: 397,\n 3098: 398,\n 3100: 399,\n 3104: 400,\n 3129: 401,\n 3157: 402,\n 3168: 403,\n 3177: 404,\n 3178: 405,\n 3179: 406,\n 3181: 407,\n 3182: 408,\n 3186: 409,\n 3196: 410,\n 3219: 411,\n 3244: 412,\n 3246: 413,\n 3250: 414,\n 3254: 415,\n 3259: 416,\n 3263: 417,\n 3264: 418,\n 3267: 419,\n 3268: 420,\n 3269: 421,\n 3286: 422,\n 3296: 423,\n 3298: 424,\n 3316: 425,\n 3327: 426,\n 3355: 427,\n 3360: 428,\n 3364: 429,\n 3409: 430,\n 3430: 431,\n 3435: 432,\n 3438: 433,\n 3450: 434,\n 3451: 435,\n 3452: 436,\n 3476: 437,\n 3477: 438,\n 3479: 439,\n 3494: 440,\n 3503: 441,\n 3504: 442,\n 3507: 443,\n 3512: 444,\n 3519: 445,\n 3525: 446,\n 3526: 447,\n 3534: 448,\n 3536: 449,\n 3546: 450,\n 3551: 451,\n 3565: 452,\n 3566: 453,\n 3594: 454,\n 3598: 455,\n 3606: 456,\n 3633: 457,\n 3638: 458,\n 3646: 459,\n 3683: 460,\n 3686: 461,\n 3693: 462,\n 3700: 463,\n 3704: 464,\n 3706: 465,\n 3708: 466,\n 3710: 467,\n 3723: 468,\n 3730: 469,\n 3743: 470,\n 3744: 471,\n 3745: 472,\n 3764: 473,\n 3783: 474,\n 3791: 475,\n 3802: 476,\n 3811: 477,\n 3821: 478,\n 3857: 479,\n 3861: 480,\n 3864: 481,\n 3879: 482,\n 3882: 483,\n 3901: 484,\n 3917: 485,\n 3918: 486,\n 3964: 487,\n 3968: 488,\n 3973: 489,\n 3983: 490,\n 3984: 491,\n 3986: 492,\n 3990: 493,\n 3992: 494,\n 3993: 495,\n 3999: 496,\n 4007: 497,\n 4015: 498,\n 4016: 499,\n 4017: 500,\n 4019: 501,\n 4020: 502,\n 4029: 503,\n 4039: 504,\n 4041: 505,\n 4054: 506,\n 4056: 507,\n 4067: 508,\n 4105: 509,\n 4144: 510,\n 4149: 511,\n 4161: 512,\n 4167: 513,\n 4184: 514,\n 4210: 515,\n 4228: 516,\n 4232: 517,\n 4235: 518,\n 4238: 519,\n 4251: 520,\n 4296: 521,\n 4318: 522,\n 4327: 523,\n 4343: 524,\n 4345: 525,\n 4378: 526,\n 4387: 527,\n 4467: 528,\n 4480: 529,\n 4492: 530,\n 4499: 531,\n 4544: 532,\n 4558: 533,\n 4623: 534,\n 4636: 535,\n 4642: 536,\n 4677: 537,\n 4679: 538,\n 4681: 539,\n 4688: 540,\n 4699: 541,\n 4700: 542,\n 4701: 543,\n 4727: 544,\n 4744: 545,\n 4784: 546,\n 4799: 547,\n 4815: 548,\n 4823: 549,\n 4833: 550,\n 4846: 551,\n 4857: 552,\n 4866: 553,\n 4873: 554,\n 4876: 555,\n 4887: 556,\n 4889: 557,\n 4898: 558,\n 4902: 559,\n 4954: 560,\n 4976: 561,\n 4980: 562,\n 4994: 563,\n 5015: 564,\n 5047: 565,\n 5049: 566,\n 5064: 567,\n 5065: 568,\n 5110: 569,\n 5111: 570,\n 5135: 571,\n 5147: 572,\n 5171: 573,\n 5219: 574,\n 5225: 575,\n 5246: 576,\n 5267: 577,\n 5283: 578,\n 5293: 579,\n 5308: 580,\n 5316: 581,\n 5339: 582,\n 5400: 583,\n 5401: 584,\n 5419: 585,\n 5420: 586,\n 5452: 587,\n 5463: 588,\n 5479: 589,\n 5504: 590,\n 5505: 591,\n 5506: 592,\n 5507: 593,\n 5530: 594,\n 5603: 595,\n 5608: 596,\n 5617: 597,\n 5620: 598,\n 5666: 599,\n 5668: 600,\n 5670: 601,\n 5690: 602,\n 5782: 603,\n 5785: 604,\n 5791: 605,\n 5812: 606,\n 5872: 607,\n 5938: 608,\n 5945: 609,\n 5954: 610,\n 5969: 611,\n 5971: 612,\n 6006: 613,\n 6078: 614,\n 6155: 615,\n 6156: 616,\n 6166: 617,\n 6184: 618,\n 6188: 619,\n 6214: 620,\n 6215: 621,\n 6219: 622,\n 6266: 623,\n 6281: 624,\n 6287: 625,\n 6294: 626,\n 6350: 627,\n 6379: 628,\n 6380: 629,\n 6383: 630,\n 6385: 631,\n 6436: 632,\n 6482: 633,\n 6538: 634,\n 6541: 635,\n 6564: 636,\n 6565: 637,\n 6593: 638,\n 6659: 639,\n 6709: 640,\n 6754: 641,\n 6764: 642,\n 6777: 643,\n 6800: 644,\n 6880: 645,\n 6888: 646,\n 6890: 647,\n 6966: 648,\n 6993: 649,\n 7004: 650,\n 7046: 651,\n 7099: 652,\n 7137: 653,\n 7149: 654,\n 7154: 655,\n 7156: 656,\n 7160: 657,\n 7162: 658,\n 7173: 659,\n 7228: 660,\n 7255: 661,\n 7265: 662,\n 7285: 663,\n 7323: 664,\n 7346: 665,\n 7360: 666,\n 7387: 667,\n 7439: 668,\n 7444: 669,\n 7445: 670,\n 7451: 671,\n 7566: 672,\n 7569: 673,\n 7618: 674,\n 7698: 675,\n 7757: 676,\n 7766: 677,\n 8014: 678,\n 8132: 679,\n 8228: 680,\n 8366: 681,\n 8507: 682,\n 8528: 683,\n 8581: 684,\n 8589: 685,\n 8638: 686,\n 8641: 687,\n 8666: 688,\n 8783: 689,\n 8807: 690,\n 8810: 691,\n 8917: 692,\n 8948: 693,\n 8969: 694,\n 8983: 695,\n 26614: 696,\n 26776: 697,\n 27611: 698,\n 27721: 699,\n 27772: 700,\n 27831: 701,\n 27873: 702,\n 27899: 703,\n 30820: 704,\n 30825: 705,\n 30850: 706,\n 31364: 707,\n 31410: 708,\n 31427: 709,\n 31658: 710,\n 31685: 711,\n 31804: 712,\n 32596: 713,\n 33615: 714,\n 33660: 715,\n 34319: 716,\n 34323: 717,\n 34437: 718,\n 34542: 719,\n 36519: 720,\n 36525: 721,\n 37729: 722,\n 37741: 723,\n 37857: 724,\n 38038: 725,\n 38061: 726,\n 38304: 727,\n 38886: 728,\n 39183: 729,\n 39292: 730,\n 39427: 731,\n 39444: 732,\n 40278: 733,\n 40339: 734,\n 40614: 735,\n 40826: 736,\n 40851: 737,\n 41573: 738,\n 41997: 739,\n 42015: 740,\n 42725: 741,\n 43560: 742,\n 44199: 743,\n 44397: 744,\n 45062: 745,\n 45210: 746,\n 45447: 747,\n 45501: 748,\n 45517: 749,\n 45672: 750,\n 45720: 751,\n 45728: 752,\n 45730: 753,\n 45950: 754,\n 46967: 755,\n 46970: 756,\n 47044: 757,\n 47200: 758,\n 47640: 759,\n 47997: 760,\n 48412: 761,\n 49649: 762,\n 50798: 763,\n 51575: 764,\n 51931: 765,\n 51935: 766,\n 52604: 767,\n 52722: 768,\n 53121: 769,\n 53125: 770,\n 53519: 771,\n 53550: 772,\n 53972: 773,\n 54001: 774,\n 54272: 775,\n 54281: 776,\n 55052: 777,\n 55276: 778,\n 55280: 779,\n 55442: 780,\n 55444: 781,\n 55765: 782,\n 56152: 783,\n 56251: 784,\n 56757: 785,\n 56805: 786,\n 56921: 787,\n 57669: 788,\n 58803: 789,\n 59118: 790,\n 59369: 791,\n 59501: 792,\n 59615: 793,\n 59900: 794,\n 60126: 795,\n 60753: 796,\n 60756: 797,\n 61132: 798,\n 62956: 799,\n 63131: 800,\n 63876: 801,\n 64839: 802,\n 64969: 803,\n 64983: 804,\n 66097: 805,\n 66297: 806,\n 66934: 807,\n 67255: 808,\n 68159: 809,\n 68237: 810,\n 68319: 811,\n 71211: 812,\n 71379: 813,\n 71535: 814,\n 72011: 815,\n 72226: 816,\n 72737: 817,\n 74458: 818,\n 76251: 819,\n 78469: 820,\n 78499: 821,\n 80463: 822,\n 81562: 823,\n 81788: 824,\n 81845: 825,\n 81847: 826,\n 81932: 827,\n 82459: 828,\n 84152: 829,\n 85510: 830,\n 86190: 831,\n 87485: 832,\n 88163: 833,\n 89804: 834,\n 89904: 835,\n 90531: 836,\n 91535: 837,\n 91542: 838,\n 91658: 839,\n 92420: 840,\n 93840: 841,\n 95510: 842,\n 96079: 843,\n 96610: 844,\n 96811: 845,\n 97921: 846,\n 98809: 847,\n 102445: 848,\n 103249: 849,\n 103335: 850,\n 103341: 851,\n 104879: 852,\n 104913: 853,\n 105844: 854,\n 112623: 855}"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_new_id_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen1hot(df):\n",
    "    ret = np.zeros((len(df), len(all_genres)), dtype = bool)\n",
    "    for index, (movie_id, genres) in df.iterrows(): \n",
    "        # print( (index, movie_id, genres))\n",
    "        for g in genres.split('|'):\n",
    "            ret[index][all_genres_map[g]] = 1\n",
    "    return ret \n",
    "movie_new_genres_1hot = gen1hot(df_new)\n",
    "movie_old_genres_1hot = gen1hot(df_old)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "array([False, False, False, False,  True,  True, False, False, False,\n       False, False, False, False, False,  True, False, False, False,\n       False])"
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_old_genres_1hot[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "movie_id                       4\ngenres      Comedy|Drama|Romance\nName: 0, dtype: object"
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_old.iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "(2610, 856)"
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "up = np.dot(movie_old_genres_1hot, movie_new_genres_1hot.T)\n",
    "up.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "array([3, 2, 1, ..., 2, 3, 2])"
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_old_genres_1hot.sum(axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "old_sum = movie_old_genres_1hot.sum(axis = 1)\n",
    "new_sum = movie_new_genres_1hot.sum(axis = 1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "[1. 1. 1. 2. 1. 2. 2. 1. 1. 2. 1. 2. 0. 1. 2. 1. 1. 1. 1. 1. 2. 0. 1. 2.\n 1. 1. 0. 1. 1. 1. 1. 2. 3. 0. 2. 1. 2. 2. 1. 1. 0. 2. 1. 2. 1. 2. 1. 1.\n 1. 0. 1. 1. 1. 2. 0. 0. 0. 0. 1. 1. 1. 0. 2. 1. 1. 2. 1. 2. 0. 2. 1. 2.\n 1. 1. 1. 0. 2. 1. 3. 1. 0. 0. 0. 2. 2. 1. 1. 1. 0. 1. 1. 0. 2. 1. 0. 1.\n 0. 0. 1. 0. 1. 0. 3. 2. 3. 0. 1. 2. 1. 1. 1. 2. 1. 1. 1. 1. 1. 0. 0. 1.\n 1. 1. 0. 0. 0. 1. 1. 2. 2. 2. 1. 2. 0. 1. 1. 2. 1. 1. 1. 1. 1. 0. 1. 2.\n 1. 1. 0. 1. 1. 1. 1. 1. 1. 1. 0. 1. 1. 2. 0. 2. 3. 1. 1. 0. 2. 1. 1. 1.\n 0. 0. 2. 1. 2. 1. 1. 1. 1. 1. 1. 2. 1. 2. 1. 1. 0. 0. 2. 2. 1. 1. 1. 2.\n 0. 1. 0. 2. 0. 1. 1. 1. 1. 1. 1. 2. 0. 2. 1. 1. 1. 2. 2. 0. 1. 0. 2. 1.\n 1. 1. 1. 1. 3. 2. 2. 1. 2. 1. 1. 1. 1. 2. 1. 1. 0. 0. 0. 0. 1. 1. 1. 2.\n 0. 1. 0. 1. 1. 2. 1. 1. 1. 0. 1. 0. 1. 1. 0. 0. 0. 1. 0. 1. 1. 1. 2. 1.\n 0. 0. 1. 0. 1. 1. 2. 3. 1. 2. 2. 1. 2. 1. 1. 2. 3. 1. 1. 1. 1. 1. 1. 0.\n 1. 1. 1. 1. 2. 2. 0. 0. 1. 0. 0. 2. 1. 1. 1. 1. 2. 1. 1. 1. 1. 2. 1. 1.\n 0. 2. 0. 1. 1. 1. 1. 0. 0. 1. 0. 0. 2. 0. 1. 1. 1. 2. 2. 0. 1. 1. 1. 0.\n 0. 1. 1. 1. 0. 1. 1. 0. 2. 2. 2. 1. 1. 2. 1. 2. 1. 1. 1. 0. 2. 1. 0. 1.\n 1. 0. 2. 0. 2. 1. 2. 1. 2. 1. 1. 2. 1. 2. 2. 1. 2. 2. 2. 2. 1. 1. 2. 1.\n 0. 0. 0. 0. 1. 0. 1. 1. 2. 1. 2. 1. 1. 3. 1. 1. 1. 2. 1. 1. 1. 1. 1. 1.\n 0. 1. 1. 0. 2. 1. 1. 1. 2. 2. 1. 0. 1. 2. 1. 1. 1. 0. 0. 0. 2. 0. 1. 1.\n 1. 1. 1. 1. 1. 0. 2. 1. 1. 1. 2. 1. 2. 1. 1. 2. 1. 3. 1. 1. 2. 2. 1. 2.\n 2. 1. 0. 1. 1. 0. 1. 1. 0. 0. 0. 1. 1. 1. 2. 0. 0. 0. 1. 1. 0. 1. 1. 0.\n 1. 0. 0. 1. 2. 0. 0. 0. 1. 0. 2. 0. 0. 1. 2. 2. 0. 1. 1. 1. 1. 1. 0. 2.\n 0. 2. 2. 1. 2. 0. 2. 2. 1. 0. 3. 1. 2. 1. 1. 0. 1. 2. 2. 0. 1. 1. 2. 0.\n 1. 0. 1. 1. 1. 1. 1. 0. 2. 1. 1. 2. 1. 2. 2. 1. 2. 0. 1. 1. 1. 2. 0. 1.\n 1. 0. 1. 0. 0. 1. 1. 1. 1. 1. 1. 3. 2. 1. 2. 2. 1. 1. 1. 2. 1. 0. 0. 2.\n 1. 1. 1. 1. 1. 1. 2. 1. 1. 1. 1. 2. 0. 1. 0. 2. 1. 0. 2. 1. 1. 3. 2. 3.\n 1. 1. 1. 1. 1. 2. 2. 0. 1. 2. 1. 3. 1. 2. 0. 2. 1. 1. 1. 1. 1. 1. 1. 3.\n 1. 1. 0. 0. 0. 0. 0. 1. 1. 1. 1. 0. 2. 1. 1. 1. 0. 0. 1. 1. 0. 0. 1. 1.\n 0. 3. 1. 1. 1. 3. 3. 2. 0. 1. 2. 2. 1. 2. 1. 1. 2. 2. 1. 1. 0. 2. 1. 1.\n 0. 0. 1. 1. 0. 1. 1. 1. 0. 2. 1. 1. 1. 1. 2. 1. 0. 1. 1. 0. 1. 3. 3. 2.\n 1. 2. 1. 2. 0. 1. 0. 2. 1. 1. 1. 1. 1. 0. 1. 2. 0. 1. 1. 2. 0. 0. 2. 0.\n 0. 2. 2. 1. 1. 1. 1. 0. 2. 2. 1. 0. 2. 1. 1. 1. 2. 0. 3. 1. 3. 1. 1. 1.\n 1. 1. 1. 1. 3. 1. 3. 2. 1. 1. 0. 1. 1. 1. 0. 1. 1. 1. 0. 1. 1. 1. 1. 1.\n 0. 1. 1. 0. 1. 0. 1. 1. 2. 2. 1. 2. 1. 1. 1. 2. 1. 1. 1. 1. 2. 2. 2. 1.\n 0. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 0. 1. 2. 1. 1. 1. 0. 2. 0. 1. 2.\n 1. 1. 1. 1. 1. 1. 1. 1. 2. 1. 2. 1. 0. 0. 0. 0. 1. 3. 1. 2. 1. 1. 1. 1.\n 0. 1. 0. 0. 0. 1. 2. 0. 0. 1. 1. 1. 1. 1. 1. 0.]\n"
    }
   ],
   "source": [
    "print(np.dot(movie_old_genres_1hot[0].astype(float) , movie_new_genres_1hot.astype(float).T ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "(2610,) (856,)\n"
    }
   ],
   "source": [
    "print(old_sum.shape, new_sum.shape )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "[[0.57735027 0.57735027 0.40824829 ... 0.40824829 0.57735027 0.        ]\n [0.         0.         0.         ... 0.         0.         0.        ]\n [0.         0.         0.         ... 0.70710678 0.         0.        ]\n ...\n [0.70710678 0.         0.5        ... 0.5        0.70710678 0.        ]\n [0.         0.         0.         ... 0.40824829 0.         0.57735027]\n [0.70710678 0.         0.5        ... 0.5        0.70710678 0.        ]]\n"
    }
   ],
   "source": [
    "down = np.array(np.sqrt([old *new_sum for old in old_sum]))\n",
    "up = np.array(np.dot(movie_old_genres_1hot.astype(float), movie_new_genres_1hot.T))\n",
    "sim_old_new = up/ down\n",
    "print(sim_old_new )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>user_id</th>\n      <th>movie_id</th>\n      <th>rating</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>1009</td>\n      <td>3.5</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>1243</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1</td>\n      <td>1848</td>\n      <td>3.5</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1</td>\n      <td>1920</td>\n      <td>3.5</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1</td>\n      <td>2118</td>\n      <td>4.0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2833278</th>\n      <td>138493</td>\n      <td>59784</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>2833279</th>\n      <td>138493</td>\n      <td>61160</td>\n      <td>4.0</td>\n    </tr>\n    <tr>\n      <th>2833280</th>\n      <td>138493</td>\n      <td>65682</td>\n      <td>4.5</td>\n    </tr>\n    <tr>\n      <th>2833281</th>\n      <td>138493</td>\n      <td>69526</td>\n      <td>4.5</td>\n    </tr>\n    <tr>\n      <th>2833282</th>\n      <td>138493</td>\n      <td>69644</td>\n      <td>3.0</td>\n    </tr>\n  </tbody>\n</table>\n<p>2833283 rows × 3 columns</p>\n</div>",
      "text/plain": "         user_id  movie_id  rating\n0              1      1009     3.5\n1              1      1243     3.0\n2              1      1848     3.5\n3              1      1920     3.5\n4              1      2118     4.0\n...          ...       ...     ...\n2833278   138493     59784     5.0\n2833279   138493     61160     4.0\n2833280   138493     65682     4.5\n2833281   138493     69526     4.5\n2833282   138493     69644     3.0\n\n[2833283 rows x 3 columns]"
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_rating_old = pd.read_csv('data/rating_old.csv')\n",
    "df_rating_old"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prefetch index to df \n",
    "df_rating_old['movie_index']  = df_rating_old.movie_id.apply(lambda _id: df_old_id_map[_id])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>user_id</th>\n      <th>movie_id</th>\n      <th>rating</th>\n      <th>movie_index</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>1217</td>\n      <td>3.5</td>\n      <td>148</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>1348</td>\n      <td>3.5</td>\n      <td>168</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1</td>\n      <td>1350</td>\n      <td>3.5</td>\n      <td>169</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1</td>\n      <td>2138</td>\n      <td>4.0</td>\n      <td>259</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1</td>\n      <td>2143</td>\n      <td>4.0</td>\n      <td>261</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>1520347</th>\n      <td>138493</td>\n      <td>45447</td>\n      <td>2.5</td>\n      <td>747</td>\n    </tr>\n    <tr>\n      <th>1520348</th>\n      <td>138493</td>\n      <td>45517</td>\n      <td>2.5</td>\n      <td>749</td>\n    </tr>\n    <tr>\n      <th>1520349</th>\n      <td>138493</td>\n      <td>53125</td>\n      <td>3.0</td>\n      <td>770</td>\n    </tr>\n    <tr>\n      <th>1520350</th>\n      <td>138493</td>\n      <td>56757</td>\n      <td>3.0</td>\n      <td>785</td>\n    </tr>\n    <tr>\n      <th>1520351</th>\n      <td>138493</td>\n      <td>68319</td>\n      <td>4.5</td>\n      <td>811</td>\n    </tr>\n  </tbody>\n</table>\n<p>1520352 rows × 4 columns</p>\n</div>",
      "text/plain": "         user_id  movie_id  rating  movie_index\n0              1      1217     3.5          148\n1              1      1348     3.5          168\n2              1      1350     3.5          169\n3              1      2138     4.0          259\n4              1      2143     4.0          261\n...          ...       ...     ...          ...\n1520347   138493     45447     2.5          747\n1520348   138493     45517     2.5          749\n1520349   138493     53125     3.0          770\n1520350   138493     56757     3.0          785\n1520351   138493     68319     4.5          811\n\n[1520352 rows x 4 columns]"
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# do the same to the new one \n",
    "df_rating_new = pd.read_csv('data/rating_new.csv')\n",
    "df_rating_new['movie_index']  = df_rating_new.movie_id.apply(lambda _id: df_new_id_map[_id])\n",
    "df_rating_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_recommend = {}\n",
    "for uid, groupby_uid in df_rating_old.groupby('user_id'):\n",
    "    movie_rating = groupby_uid.groupby('movie_index').rating.mean().sort_values(ascending = False)\n",
    "    fav = movie_rating[movie_rating >= 4]\n",
    "    # print(fav.index)\n",
    "    # print (np.where(sim_old_new[fav.index.tolist()] >= 0.85)[1] )\n",
    "    user_recommend[uid] =set(np.unique(np.where(sim_old_new[fav.index.tolist()] >= 0.85)[1] )[:100])\n",
    "    # break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "('precision', 0.06963247644471261)\n('recall', 0.17295069126066484)\n"
    }
   ],
   "source": [
    "\n",
    "ac_quantity = 0\n",
    "rec_quantity = 0 \n",
    "fav_quantity = 0\n",
    "\n",
    "for uid, grouby_uid  in df_rating_new.groupby('user_id'):\n",
    "    movies_rating = grouby_uid.groupby('movie_index').rating.mean()\n",
    "    # print(movies_rating)\n",
    "    # print(movies_rating[movies_rating > 2.5].index)\n",
    "    user_fav = set(movies_rating[movies_rating > 2.5].index)\n",
    "    fav_quantity += len(user_fav)\n",
    "    if uid in user_recommend: \n",
    "        ac_quantity += len(user_recommend[uid] & user_fav)\n",
    "        rec_quantity +=  len(user_recommend[uid])\n",
    "    # break\n",
    "\n",
    "print(('precision', ac_quantity/ rec_quantity))\n",
    "print(('recall', ac_quantity/fav_quantity))"
   ]
  }
 ]
}